{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/xiangjx/anaconda3/envs/musk3/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import os\n",
    "import subprocess\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"4\"\n",
    "import huggingface_hub\n",
    "huggingface_hub.login('')  # get your own HF token\n",
    "\n",
    "# Function to load and display JSON results\n",
    "def load_and_print_results(file_path):\n",
    "    if os.path.exists(file_path):\n",
    "        # Read the file\n",
    "        with open(file_path) as f:\n",
    "            # Read each line and decode it as a separate JSON object\n",
    "            for line in f:\n",
    "                try:\n",
    "                    data = json.loads(line)\n",
    "                    print(data)\n",
    "                except json.JSONDecodeError as e:\n",
    "                    print(f\"Error decoding JSON: {e}\")\n",
    "                    \n",
    "    else:\n",
    "        print(f\"No results found at {file_path}\")\n",
    "\n",
    "# Function to run a command and print output\n",
    "def run_command(command):\n",
    "    result = subprocess.run(command, shell=True, capture_output=True, text=True)\n",
    "    print(result.stdout)\n",
    "    if result.stderr:\n",
    "        print(result.stderr)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Zero-shot cross-modal retrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "command = (\n",
    "    \"python3 -m clip_benchmark.cli eval --pretrained_model models.txt \"\n",
    "    \"--dataset 'pathmmu_retrieval' \"\n",
    "    \"--task 'zeroshot_retrieval' \"\n",
    "    \"--batch_size 128 \"\n",
    "    \"--num_workers 16 \"\n",
    "    \"--seed 42 \"\n",
    "    \"--recall_k 1 10 50 \"\n",
    "    \"--dataset_root '/mnt/radonc-li01/private/xiangjx/data/downstreams_demo' \"\n",
    "    \"--output './results/benchmark_mm_retrieval.json' > /dev/null 2>&1\"\n",
    ")\n",
    "run_command(command)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'dataset': 'pathmmu_retrieval', 'model': 'musk_large_patch16_384', 'pretrained': 'hf_hub:xiangjx/musk', 'task': 'zeroshot_retrieval', 'metrics': {'image_retrieval_recall@1': 0.03087213821709156, 'text_retrieval_recall@1': 0.040648315101861954, 'image_retrieval_recall@10': 0.15731926262378693, 'text_retrieval_recall@10': 0.17661435902118683, 'image_retrieval_recall@50': 0.3395935297012329, 'text_retrieval_recall@50': 0.34435296058654785}, 'language': 'en'}\n",
      "{'dataset': 'pathmmu_retrieval', 'model': 'conch', 'pretrained': '/mnt/radonc-li01/private/xiangjx/models/conch/pytorch_model.bin', 'task': 'zeroshot_retrieval', 'metrics': {'image_retrieval_recall@1': 0.019938256591558456, 'text_retrieval_recall@1': 0.022896835580468178, 'image_retrieval_recall@10': 0.10869564861059189, 'text_retrieval_recall@10': 0.1201440691947937, 'image_retrieval_recall@50': 0.2658862769603729, 'text_retrieval_recall@50': 0.27720606327056885}, 'language': 'en'}\n"
     ]
    }
   ],
   "source": [
    "load_and_print_results('./results/benchmark_mm_retrieval.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Zero-shot classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "command = (\n",
    "    \"python3 -m clip_benchmark.cli eval --pretrained_model models.txt \"\n",
    "    \"--dataset 'skin' 'pannuke' 'unitopatho' \"\n",
    "    \"--task 'zeroshot_classification' \"\n",
    "    \"--batch_size 256 \"\n",
    "    \"--dataset_root '/mnt/radonc-li01/private/xiangjx/data/downstreams_demo' \"\n",
    "    \"--output './results/zeroshot_cls.json' > /dev/null 2>&1\"\n",
    ")\n",
    "run_command(command)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'dataset': 'skin', 'model': 'musk_large_patch16_384', 'pretrained': 'hf_hub:xiangjx/musk', 'task': 'zeroshot_classification', 'metrics': {'balanced_acc': 0.5878613953752625}, 'language': 'en'}\n",
      "{'dataset': 'pannuke', 'model': 'musk_large_patch16_384', 'pretrained': 'hf_hub:xiangjx/musk', 'task': 'zeroshot_classification', 'metrics': {'balanced_acc': 0.8280807628343463}, 'language': 'en'}\n",
      "{'dataset': 'unitopatho', 'model': 'musk_large_patch16_384', 'pretrained': 'hf_hub:xiangjx/musk', 'task': 'zeroshot_classification', 'metrics': {'balanced_acc': 0.368094020538031}, 'language': 'en'}\n",
      "{'dataset': 'skin', 'model': 'conch', 'pretrained': '/mnt/radonc-li01/private/xiangjx/models/conch/pytorch_model.bin', 'task': 'zeroshot_classification', 'metrics': {'balanced_acc': 0.21905265325279444}, 'language': 'en'}\n",
      "{'dataset': 'pannuke', 'model': 'conch', 'pretrained': '/mnt/radonc-li01/private/xiangjx/models/conch/pytorch_model.bin', 'task': 'zeroshot_classification', 'metrics': {'balanced_acc': 0.5703599853491457}, 'language': 'en'}\n",
      "{'dataset': 'unitopatho', 'model': 'conch', 'pretrained': '/mnt/radonc-li01/private/xiangjx/models/conch/pytorch_model.bin', 'task': 'zeroshot_classification', 'metrics': {'balanced_acc': 0.3023926864066804}, 'language': 'en'}\n"
     ]
    }
   ],
   "source": [
    "load_and_print_results('./results/zeroshot_cls.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Zero-shot Image2Image Retrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# image-to-image retrieval\n",
    "command = (\n",
    "    \"python3 -m clip_benchmark.cli eval --pretrained_model models.txt \"\n",
    "    \"--dataset 'unitopatho_retrieval' \"\n",
    "    \"--task 'image_retrieval' \"\n",
    "    \"--batch_size 128 \"\n",
    "    \"--num_workers 8 \"\n",
    "    \"--seed 41 \"\n",
    "    \"--dataset_root '/mnt/radonc-li01/private/xiangjx/data/downstreams_demo' \"\n",
    "    \"--output './results/image_retrieval.json' > /dev/null 2>&1\"\n",
    ")\n",
    "run_command(command)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'dataset': 'unitopatho_retrieval', 'model': 'musk_large_patch16_384', 'pretrained': 'hf_hub:xiangjx/musk', 'task': 'image_retrieval', 'metrics': {'acc_top1': 0.8968120813369751, 'acc_top3': 0.9610947966575623, 'acc_top5': 0.9767197966575623, 'mMv_top5': 0.8673447966575623}, 'language': 'en'}\n",
      "{'dataset': 'unitopatho_retrieval', 'model': 'conch', 'pretrained': '/mnt/radonc-li01/private/xiangjx/models/conch/pytorch_model.bin', 'task': 'image_retrieval', 'metrics': {'acc_top1': 0.833787739276886, 'acc_top3': 0.9383389353752136, 'acc_top5': 0.9625629186630249, 'mMv_top5': 0.8200503587722778}, 'language': 'en'}\n"
     ]
    }
   ],
   "source": [
    "load_and_print_results('./results/image_retrieval.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 10-shot classification\n",
    "\n",
    "It will take long time to extract features and run supervised cls."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Few-shot linear probe commands\n",
    "seed = 123\n",
    "k_shot = 10\n",
    "\n",
    "command = (\n",
    "    f\"python3 -m clip_benchmark.cli eval --pretrained_model models.txt \"\n",
    "    f\"--dataset  'skin' 'unitopatho' 'pannuke' \"\n",
    "    f\"--task 'linear_probe' \"\n",
    "    f\"--batch_size 256 \"\n",
    "    f\"--num_workers 8 \"\n",
    "    f\"--fewshot_k {k_shot} \"\n",
    "    f\"--seed {seed} \"\n",
    "    f\"--dataset_root '/mnt/radonc-li01/private/xiangjx/data/downstreams_demo' \"\n",
    "    f\"--output './results/benchmark_fs_{k_shot}shot_seed{seed}.json' > /dev/null 2>&1\"\n",
    ")\n",
    "\n",
    "run_command(command)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'dataset': 'skin', 'model': 'musk_large_patch16_384', 'pretrained': 'hf_hub:xiangjx/musk', 'task': 'linear_probe', 'metrics': {'balanced_acc': 0.898394069992517}, 'language': 'en'}\n",
      "{'dataset': 'unitopatho', 'model': 'musk_large_patch16_384', 'pretrained': 'hf_hub:xiangjx/musk', 'task': 'linear_probe', 'metrics': {'balanced_acc': 0.5591443143634388}, 'language': 'en'}\n",
      "{'dataset': 'pannuke', 'model': 'musk_large_patch16_384', 'pretrained': 'hf_hub:xiangjx/musk', 'task': 'linear_probe', 'metrics': {'balanced_acc': 0.8010187540834307}, 'language': 'en'}\n",
      "{'dataset': 'skin', 'model': 'conch', 'pretrained': '/mnt/radonc-li01/private/xiangjx/models/conch/pytorch_model.bin', 'task': 'linear_probe', 'metrics': {'balanced_acc': 0.8813451499159908}, 'language': 'en'}\n",
      "{'dataset': 'unitopatho', 'model': 'conch', 'pretrained': '/mnt/radonc-li01/private/xiangjx/models/conch/pytorch_model.bin', 'task': 'linear_probe', 'metrics': {'balanced_acc': 0.5176345673715952}, 'language': 'en'}\n",
      "{'dataset': 'pannuke', 'model': 'conch', 'pretrained': '/mnt/radonc-li01/private/xiangjx/models/conch/pytorch_model.bin', 'task': 'linear_probe', 'metrics': {'balanced_acc': 0.7644163663901482}, 'language': 'en'}\n"
     ]
    }
   ],
   "source": [
    "load_and_print_results(f'./results/benchmark_fs_{k_shot}shot_seed{seed}.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear probe classification\n",
    "\n",
    "It will take long time to extract features and run supervised cls."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Linear probe commands\n",
    "seed = 123\n",
    "\n",
    "os.system(\"rm -rf features/\")\n",
    "\n",
    "command = (\n",
    "    f\"python3 -m clip_benchmark.cli eval --pretrained_model models.txt \"\n",
    "    f\"--dataset 'skin' 'unitopatho' 'pannuke' \"\n",
    "    f\"--task 'linear_probe' \"\n",
    "    f\"--batch_size 256 \"\n",
    "    f\"--num_workers 8 \"\n",
    "    f\"--fewshot_k -1 \"\n",
    "    f\"--seed {seed} \"\n",
    "    f\"--dataset_root '/mnt/radonc-li01/private/xiangjx/data/downstreams_demo' \"\n",
    "    f\"--ms_aug \"\n",
    "    f\"--output './results/linear_probe_{seed}.json' > /dev/null 2>&1 \"\n",
    ")\n",
    "\n",
    "run_command(command)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'dataset': 'skin', 'model': 'musk_large_patch16_384', 'pretrained': 'hf_hub:xiangjx/musk', 'task': 'linear_probe', 'metrics': {'balanced_acc': 0.9582715967801188}, 'language': 'en'}\n",
      "{'dataset': 'unitopatho', 'model': 'musk_large_patch16_384', 'pretrained': 'hf_hub:xiangjx/musk', 'task': 'linear_probe', 'metrics': {'balanced_acc': 0.7796039693199256}, 'language': 'en'}\n",
      "{'dataset': 'pannuke', 'model': 'musk_large_patch16_384', 'pretrained': 'hf_hub:xiangjx/musk', 'task': 'linear_probe', 'metrics': {'balanced_acc': 0.9604754845671069}, 'language': 'en'}\n",
      "{'dataset': 'skin', 'model': 'conch', 'pretrained': '/mnt/radonc-li01/private/xiangjx/models/conch/pytorch_model.bin', 'task': 'linear_probe', 'metrics': {'balanced_acc': 0.9447268567940842}, 'language': 'en'}\n",
      "{'dataset': 'unitopatho', 'model': 'conch', 'pretrained': '/mnt/radonc-li01/private/xiangjx/models/conch/pytorch_model.bin', 'task': 'linear_probe', 'metrics': {'balanced_acc': 0.7383896813889964}, 'language': 'en'}\n",
      "{'dataset': 'pannuke', 'model': 'conch', 'pretrained': '/mnt/radonc-li01/private/xiangjx/models/conch/pytorch_model.bin', 'task': 'linear_probe', 'metrics': {'balanced_acc': 0.9478886188600051}, 'language': 'en'}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "load_and_print_results(f'./results/linear_probe_{seed}.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "musk3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
